package com.crawler.jsoup;

import com.alibaba.excel.EasyExcel;
import com.crawler.jsoup.model.Douban;
import org.jsoup.Connection;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * Jsoup爬取豆瓣精选
 * <p>
 * Created by devinlee in 2021.08.03
 */
public class JsoupApplication {

    //生成的附件名
    private final static String FILENAME = "今日豆瓣精选";
    //系统最后一个盘符的路径
    private final static String ROOTPATH = File.listRoots()[File.listRoots().length - 1].getPath();
    //测试账号
    private final static List<Map<String, String>> cookieMap = Arrays.asList(
            new HashMap<String, String>() {{
                put("dbcl2", "217807068:NjMBlsG4IuU");//测试号1
            }},
            new HashMap<String, String>() {{
                put("dbcl2", "181149058:z43aPQn4Pps");//测试号2
            }}
    );
    //存储豆瓣数据，包含标题、链接、图片链接、摘要、点赞数、小组来源
    private static List<Douban> doubanList = new ArrayList<>();

    private static final int pageSize = 30;//每页大小
    private static int startNum = 0;//起始索引

    private static final ThreadLocal<DateTimeFormatter> DATE_TIME_FORMATTER = new ThreadLocal<DateTimeFormatter>() {{
        set(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
    }};

    /**
     * 入口
     * <p>
     *
     * @param args
     */
    public static void main(String[] args) {

        /*********************爬取豆瓣精选*********************/
        try {
            JsoupApplication.doCrawler();//执行爬取
        } catch (SocketTimeoutException e) {
            System.err.println("\n网络发生异常：错误信息[" + e.getMessage() + "]");
            System.out.println("正在尝试重新连接...");
            JsoupApplication.main(new String[]{});
        } catch (HttpStatusException e) {
            System.err.println("\n网络发生异常：错误信息[" + e.getMessage() + "]");
            System.out.println("正在尝试重新连接...");
            JsoupApplication.main(new String[]{});
        } catch (ConnectException e) {
            System.err.println("\n网络发生异常：连接失败");
            return;//结束进程
        } catch (IOException e) {
            e.printStackTrace();
            System.err.println("\nIO发生异常：错误信息[" + e.getMessage() + "]");
            return;
        } catch (Exception e) {
            e.printStackTrace();
            System.err.println("\n发生异常：错误信息[" + e.getMessage() + "]");
            return;
        }


        /**********************写入excel**********************/
        try {
            String filePath = String.format("%s%s%s.xlsx", JsoupApplication.ROOTPATH, JsoupApplication.FILENAME, new SimpleDateFormat("yyyyMMdd").format(new Date()));
            File file = new File(filePath);
            if (!file.exists()) {
                file.createNewFile();//创建空文件
            }
            if (!file.canWrite()) {
                throw new IOException("文件不可写入！");
            }
            //写入磁盘(按点赞数排序)
            EasyExcel.write(filePath, Douban.class).sheet(JsoupApplication.FILENAME).doWrite(
                    doubanList.stream()
                            .sorted(Comparator.comparing(Douban::getLike).reversed())
                            .collect(Collectors.toList())
            );
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("IO发生异常：错误信息[" + e.getMessage() + "]");
        }

        System.out.println("=========[END]========");
    }

    /**
     * 爬取并解析逻辑(如异常将接收并重试)
     * <p>
     *
     * @throws IOException
     * @throws SocketTimeoutException
     * @throws HttpStatusException
     */
    private static void doCrawler() throws IOException, SocketTimeoutException, HttpStatusException, ConnectException {

        StringBuilder url = new StringBuilder();
        while (true) {
            //创建豆瓣链接
            Connection connection = Jsoup.connect(url.delete(0, url.length()).append("https://www.douban.com/group/explore?start=").append(startNum).toString());
            //设置用户代理，模拟浏览器请求
            connection.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
            //设置cookie模拟登录（浏览器登录获取）
            connection.cookies(cookieMap.get(new Random().nextInt(cookieMap.size())));//随机取一条测试号
            //设置代理（加入多节点网关）
            connection.proxy("127.0.0.1", 1080);

            System.out.print(String.format("[%s]正在请求url:%s", LocalDateTime.now().format(JsoupApplication.DATE_TIME_FORMATTER.get()), url.toString()));
            //发送请求并接收Document对象
            Document document = connection.get();
            System.out.println(String.format("[返回码：%d]", connection.response().statusCode()));

            //解析html
            Elements elements = document.select("#content > div > div.article > div:nth-child(1) > div");//定位列表元素
            //提取点赞数正则
            Pattern patternLike = Pattern.compile("[^0-9]");
            elements.stream().forEach(element -> {
                doubanList.add(new Douban(
                        element.getElementsByTag("a").get(0).text(),
                        element.getElementsByTag("a").first().attr("href"),
                        element.getElementsByTag("img").size() > 0 ? element.getElementsByTag("img").first().attr("src") : "",
                        element.select(".bd > .block > p").first().text(),
                        Integer.valueOf(patternLike.matcher(element.getElementsByClass("likes").first().text()).replaceAll("")),
                        element.select(".source > .from").first().getElementsByTag("a").first().text()
                ));
            });
            //检查是否还有下一页
            if (elements.size() <= 0) {
                System.out.println(String.format("[%s]返回无结果，请求结束！", LocalDateTime.now().format(DATE_TIME_FORMATTER.get())));
                break;
            }
            //累加下一页参数
            startNum += pageSize;
        }
    }
}
